{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "### 汉字拼音编码\n",
    "\n",
    "[pypinyin 0.39.1 文档](https://pypinyin.readthedocs.io/zh_CN/master/usage.html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pypinyin  Version:  0.39.1\n"
     ]
    }
   ],
   "source": [
    "import pypinyin\n",
    "from pypinyin import pinyin, lazy_pinyin, Style\n",
    "print('pypinyin  Version:  {}'.format(pypinyin.__version__))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['nǐ'], ['hǎo'], ['zhōng'], ['guó'], ['*'], ['*'], ['a']]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 错误字符以*号代替\n",
    "pinyin('你好中国☆☆啊', errors=lambda x: ['*' for _ in x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['n'], ['h'], ['z'], ['g'], ['☆☆'], ['a']]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 拼音首字母\n",
    "pinyin('你好中国☆☆啊', style=Style.FIRST_LETTER)  # 设置拼音风格"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['zho4ng', 'cho2ng', 'to2ng']]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 多音词\n",
    "pinyin('重', style=Style.TONE2, heteronym=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['ch'], ['q'], ['☆☆']]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 声母\n",
    "pinyin('重庆☆☆', style=pypinyin.INITIALS, heteronym=False, strict=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['v4']]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 韵母、声调\n",
    "pinyin('绿', style=pypinyin.FINALS_TONE3, heteronym=False, strict=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ni', 'hao', 'zhong', 'guo', '☆☆', 'a']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 不考虑多音字的情况， 无声调\n",
    "lazy_pinyin('你好中国☆☆啊')  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# help(pypinyin)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "### 汉字字形编码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "D:\\Desktop\\新建文件夹\\NoteBooks\\NLP\\zh_data\\Strokes.txt\n",
      "hz_strokes_dict  汉字笔画数量：20921\n",
      "卝  4\n",
      "羋  8\n",
      "羐  10\n",
      "瑴  14\n",
      "彛  16\n",
      "龜  16\n",
      "彞  18\n",
      "龞  27\n",
      "丨  1\n",
      "丩  2\n",
      "丬  3\n",
      "丫  3\n"
     ]
    }
   ],
   "source": [
    "import os \n",
    "\n",
    "# 汉字笔划\n",
    "strokes_file_path = os.path.join('.', 'zh_data', 'Strokes.txt')\n",
    "print(os.path.abspath(strokes_file_path))\n",
    "\n",
    "# 汉字笔划字典\n",
    "hz_strokes_dict = dict()\n",
    "\n",
    "for line_i in open(strokes_file_path, 'r', encoding='utf-8'):\n",
    "    line_i = line_i.split()\n",
    "    hz_strokes_dict[line_i[1]] = line_i[2]\n",
    "    \n",
    "print('hz_strokes_dict  汉字笔画数量：{}'.format(len(hz_strokes_dict)))\n",
    "\n",
    "for i, (k, v) in enumerate(hz_strokes_dict.items()):\n",
    "    print('{}  {}'.format(k, v))\n",
    "    if i > 10:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "D:\\Desktop\\新建文件夹\\NoteBooks\\NLP\\zh_data\\Structure.txt\n",
      "hz_structure_dict  汉字形体数量：88480\n",
      "⺀  ⿱\n",
      "⺸  ⿱\n",
      "⺻  ⿱\n",
      "⺼  ⿵\n",
      "〢  ⿰\n",
      "丁  ⿱\n",
      "丂  ⿱\n",
      "七  ⿻\n",
      "丄  ⿱\n",
      "丅  ⿱\n",
      "丆  ⿱\n",
      "万  ⿱\n"
     ]
    }
   ],
   "source": [
    "# 汉字：形体结构\n",
    "structure_file_path = os.path.join('.', 'zh_data', 'Structure.txt')\n",
    "print(os.path.abspath(structure_file_path))\n",
    "\n",
    "# 汉字形体结果字典\n",
    "hz_structure_dict = dict()\n",
    "\n",
    "# 字形字典\n",
    "shapeDict = {'⿰': '1', '⿱': '2', '⿲': '3', '⿳': '4', '⿴': '5',  # 左右结构、上下、左中右、上中下、全包围\n",
    "             '⿵': '6', '⿶': '7', '⿷': '8', '⿸': '9', '⿹': 'A',  # 上三包、下三包、左三包、左上包、右上包\n",
    "             '⿺': 'B', '⿻': 'C', '0': '0'}  # 左下包、镶嵌、独体字：0\n",
    "\n",
    "for line_i in open(structure_file_path, 'r', encoding='utf-8'):\n",
    "    line_i = line_i.split()\n",
    "    if line_i[2][0] in shapeDict:\n",
    "        hz_structure_dict[line_i[1]] = line_i[2][0]\n",
    "print('hz_structure_dict  汉字形体数量：{}'.format(len(hz_structure_dict)))\n",
    "\n",
    "for i, (k, v) in enumerate(hz_structure_dict.items()):\n",
    "    print('{}  {}'.format(k, v))\n",
    "    if i > 10:\n",
    "        break\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "D:\\Desktop\\新建文件夹\\NoteBooks\\NLP\\zh_data\\FourCorner.txt\n",
      "hz_strokes_dict  汉字四角编码数量：74154\n",
      "亠:  00000\n",
      "弯:  00027\n",
      "亪:  00037\n",
      "疒:  00100\n",
      "韲:  00101\n",
      "𣁇:  00102\n",
      "𩐐:  00102\n",
      "𣁪:  00102\n",
      "𥃐:  00102\n",
      "𥃂:  00102\n",
      "𥂶:  00102\n",
      "𥂡:  00102\n"
     ]
    }
   ],
   "source": [
    "# 四角编码  为什么是5位数字编码：https://zhidao.baidu.com/question/1667714057688997667.html\n",
    "fourcorner_file_path = os.path.join('.', 'zh_data', 'FourCorner.txt')\n",
    "print(os.path.abspath(fourcorner_file_path))\n",
    "\n",
    "# 汉字笔划字典\n",
    "hz_fourcorner_dict = dict()\n",
    "\n",
    "for line_i in open(fourcorner_file_path, 'r', encoding='utf-8'):\n",
    "    line_i = line_i.split()\n",
    "    hz_fourcorner_dict[line_i[0]] = line_i[1]\n",
    "\n",
    "print('hz_strokes_dict  汉字四角编码数量：{}'.format(len(hz_fourcorner_dict)))\n",
    "\n",
    "for i, (k, v) in enumerate(hz_fourcorner_dict.items()):\n",
    "    print('{}  {}'.format(k, v))\n",
    "    if i > 10:\n",
    "        break\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "# 拼音&字形相似度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pypinyin\n",
    "\n",
    "\n",
    "# 拼音码 获取\n",
    "class SoundCode(object):\n",
    "    \"\"\"\n",
    "    计算输入文本字符串对应的拼音码\n",
    "    【声母 (23+1)】+【声调 (4+1)】+【韵母 (38+1)】\n",
    "\n",
    "    使用第三方包 pypinyin  0.39.1    https://pypinyin.readthedocs.io/zh_CN/master/usage.html\n",
    "\n",
    "    默认开启模糊音，可根据实际修改模糊音\n",
    "     {'zh': 'z', 'ch': 'c', 'sh': 's', 'h': 'f', 'n': 'l', 'r': 'l'}\n",
    "     {'ang': 'an', 'eng': 'en', 'ing': 'in', 'iang': 'ian', 'uang': 'uan'}\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, yunmu_fuzzy=True, shengmu_fuzzy=True, debug=False):\n",
    "        \"\"\"\n",
    "        :param yunmu_fuzzy:     韵母模糊音设置\n",
    "        :param shengmu_fuzzy:   声母模糊音设置\n",
    "        :param debug:           调试打印\n",
    "        \"\"\"\n",
    "        self.debug = debug\n",
    "\n",
    "        # 声母  23 个\n",
    "        self.shengmu_dict = {'b': '1', 'p': '2', 'm': '3', 'f': '4',\n",
    "                             'd': '5', 't': '6', 'n': '7', 'l': '8',\n",
    "                             'g': '9', 'k': 'A', 'h': 'B', 'j': 'C',\n",
    "                             'q': 'D', 'x': 'E', 'zh': 'F', 'ch': 'G',\n",
    "                             'sh': 'H', 'r': 'I', 'z': 'J', 'c': 'K',\n",
    "                             's': 'L', 'y': 'M', 'w': 'N'}\n",
    "\n",
    "        # 韵母  38 个\n",
    "        self.yunmu_dict = {'a': '1', 'o': '2', 'e': '3', 'i': '4', 'u': '5', 'v': '6',\n",
    "                           'ai': '7', 'ei': '8', 'ao': '9', 'ou': 'a', 'ia': 'b', 'ie': 'c',\n",
    "                           'ua': 'd', 'uo': 'e', 've': 'f', 'iao': 'g', 'iou': 'h', 'uai': 'i',\n",
    "                           'uei': 'j', 'an': 'k', 'ian': 'l', 'uan': 'm', 'van': 'n', 'en': 'o',\n",
    "                           'in': 'p', 'uen': 'q', 'vn': 'r', 'ang': 's', 'iang': 't', 'uang': 'u',\n",
    "                           'eng': 'v', 'ing': 'w', 'ueng': 'x', 'ong': 'y', 'uong': 'z', 'iong': 'A',\n",
    "                           'er': 'B', 'io': 'C'}\n",
    "\n",
    "        # 声母模糊音设置\n",
    "        if shengmu_fuzzy:\n",
    "            # 可根据实际情况进行修改模糊音\n",
    "            self.shengmu_fuzzy_tone = {'zh': 'z', 'ch': 'c', 'sh': 's', 'h': 'f', 'n': 'l', 'r': 'l'}\n",
    "\n",
    "            for f_k, f_v in self.shengmu_fuzzy_tone.items():\n",
    "                self.shengmu_dict[f_k] = self.shengmu_dict[f_v]\n",
    "\n",
    "        else:\n",
    "            self.shengmu_fuzzy_tone = {}\n",
    "\n",
    "        # 韵母模糊音设置\n",
    "        if yunmu_fuzzy:\n",
    "            # 可根据实际情况进行修改模糊音\n",
    "            self.yunmu_fuzzy_tone = {'ang': 'an', 'eng': 'en', 'ing': 'in', 'iang': 'ian', 'uang': 'uan'}\n",
    "\n",
    "            for f_k, f_v in self.yunmu_fuzzy_tone.items():\n",
    "                self.yunmu_dict[f_k] = self.yunmu_dict[f_v]\n",
    "\n",
    "        else:\n",
    "            self.yunmu_fuzzy_tone = {}\n",
    "\n",
    "    def __call__(self, words):\n",
    "        \"\"\"\n",
    "        :param words:  词字符串  '重庆人民的☆☆重要时间啊'\n",
    "        :return:\n",
    "        \"\"\"\n",
    "        # 声母\n",
    "        shen_mu_list = pypinyin.pinyin(words, style=pypinyin.INITIALS, heteronym=False, strict=False,\n",
    "                                       errors=lambda x: [None for _ in x])\n",
    "\n",
    "        # 韵母、声调\n",
    "        yun_mu_list = pypinyin.pinyin(words, style=pypinyin.FINALS_TONE3, heteronym=False, strict=True,\n",
    "                                      errors=lambda x: [None for _ in x])\n",
    "\n",
    "        # 输入词的拼音码\n",
    "        sound_codes = list()\n",
    "\n",
    "        for shen_mu_i, yun_mu_i in zip(shen_mu_list, yun_mu_list):\n",
    "            # print(shen_mu_i, yun_mu_i)\n",
    "            shengmu_str = shen_mu_i[0]\n",
    "            yun_mu_str = yun_mu_i[0]\n",
    "\n",
    "            # 声母   非正常汉字【☆】 或 不在声母列表中【啊】  声母以'0'代替\n",
    "            if shengmu_str is not None and shengmu_str in self.shengmu_dict:\n",
    "                shengmu_code = self.shengmu_dict[shengmu_str]\n",
    "            else:\n",
    "                shengmu_code = '0'\n",
    "\n",
    "            # 声调\n",
    "            if yun_mu_str is not None and yun_mu_str[-1] in ['1', '2', '3', '4']:\n",
    "                yindiao_code = yun_mu_str[-1]\n",
    "                yun_mu_str = yun_mu_str[:-1]  # 带声调的韵母去除声调\n",
    "            else:\n",
    "                yindiao_code = '0'  # 无声调 默认为 '0'\n",
    "\n",
    "            # 韵母\n",
    "            if yun_mu_str is not None and yun_mu_str in self.yunmu_dict:\n",
    "                yunmu_code = self.yunmu_dict[yun_mu_str]\n",
    "            else:\n",
    "                yunmu_code = '0'\n",
    "\n",
    "            # print('{}  {}  {} '.format(shengmu_code, yindiao_code, yunmu_code))\n",
    "            sound_codes.append(shengmu_code + yindiao_code + yunmu_code)\n",
    "\n",
    "        return sound_codes\n",
    "\n",
    "\n",
    "# 字形码 获取\n",
    "class ShapeCode(object):\n",
    "    \"\"\"\n",
    "    计算输入文本字符串对应的字形码\n",
    "    【结构编码（1位）】 + 【四角编码 （5位）】 + 【笔划编码（1位）】\n",
    "\n",
    "    参考资料\n",
    "    https://github.com/qingyujean/ssc\n",
    "    https://blog.csdn.net/chndata/article/details/41114771\n",
    "\n",
    "    为什么是5位数字编码：\n",
    "    https://zhidao.baidu.com/question/1667714057688997667.html\n",
    "\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, structure_file_path, fourcorner_file_path, strokes_file_path, debug=False):\n",
    "        \"\"\"\n",
    "        :param structure_file_path:    汉字结构编码表\n",
    "        :param fourcorner_file_path:   汉字四角编码表\n",
    "        :param strokes_file_path:      汉字笔画编码表\n",
    "        :param debug:           调试打印\n",
    "        \"\"\"\n",
    "        self.debug = debug\n",
    "        # ###############################################################################################\n",
    "        # 汉字形体字典\n",
    "        self.hz_structure_dict = dict()\n",
    "\n",
    "        # 字形编码\n",
    "        self._hz_shape_dict = {'⿰': '1', '⿱': '2', '⿲': '3', '⿳': '4', '⿴': '5',  # 左右结构、上下、左中右、上中下、全包围\n",
    "                               '⿵': '6', '⿶': '7', '⿷': '8', '⿸': '9', '⿹': 'A',  # 上三包、下三包、左三包、左上包、右上包\n",
    "                               '⿺': 'B', '⿻': 'C', '0': 'D'}  # 左下包、镶嵌、独体字：D\n",
    "\n",
    "        for line_i in open(structure_file_path, 'r', encoding='utf-8'):\n",
    "            line_i = line_i.split()\n",
    "            if line_i[2][0] in self._hz_shape_dict:\n",
    "                self.hz_structure_dict[line_i[1]] = self._hz_shape_dict[line_i[2][0]]\n",
    "            else:\n",
    "                self.hz_structure_dict[line_i[1]] = 'D'\n",
    "\n",
    "        if self.debug:\n",
    "            print('hz_structure_dict  汉字结构编码数量：{}   {}'.format(len(self.hz_structure_dict),\n",
    "                                                               os.path.abspath(structure_file_path)))\n",
    "\n",
    "        # ###############################################################################################\n",
    "        # 汉字四角编码\n",
    "        self.hz_fourcorner_dict = dict()\n",
    "\n",
    "        for line_i in open(fourcorner_file_path, 'r', encoding='utf-8'):\n",
    "            line_i = line_i.strip().split(':')\n",
    "            self.hz_fourcorner_dict[line_i[0]] = line_i[1].strip()\n",
    "\n",
    "        if self.debug:\n",
    "            print('hz_fourcorner_dict  汉字四角编码数量：{}   {}'.format(len(self.hz_fourcorner_dict),\n",
    "                                                                os.path.abspath(fourcorner_file_path)))\n",
    "\n",
    "        # ###############################################################################################\n",
    "        # 汉字笔划编码\n",
    "        self.hz_strokes_dict = dict()\n",
    "\n",
    "        # 笔画数字典\n",
    "        self.strokes_num_dict = {1: '1', 2: '2', 3: '3', 4: '4', 5: '5', 6: '6', 7: '7', 8: '8', 9: '9', 10: 'A',\n",
    "                                 11: 'B', 12: 'C', 13: 'D', 14: 'E', 15: 'F', 16: 'G', 17: 'H', 18: 'I', 19: 'J',\n",
    "                                 20: 'K',\n",
    "                                 21: 'L', 22: 'M', 23: 'N', 24: 'O', 25: 'P', 26: 'Q', 27: 'R', 28: 'S', 29: 'T',\n",
    "                                 30: 'U',\n",
    "                                 31: 'V', 32: 'W', 33: 'X', 34: 'Y', 35: 'Z', 0: '0'}\n",
    "\n",
    "        for line_i in open(strokes_file_path, 'r', encoding='utf-8'):\n",
    "            line_i = line_i.strip().split()\n",
    "            s_num = int(line_i[2])  # 笔划数\n",
    "            if s_num > 35:  # 大于 35 都统一为35\n",
    "                s_num = 35\n",
    "\n",
    "            self.hz_strokes_dict[line_i[1]] = self.strokes_num_dict[s_num]\n",
    "\n",
    "        if self.debug:\n",
    "            print('hz_strokes_dict  汉字笔画编码数量：{}   {}'.format(len(self.hz_strokes_dict),\n",
    "                                                             os.path.abspath(strokes_file_path)))\n",
    "\n",
    "    def __call__(self, words):\n",
    "        \"\"\"\n",
    "        :param words:  词字符串  '重庆人民的☆☆重要时间啊'\n",
    "        :return:\n",
    "        \"\"\"\n",
    "        # 输入词的字形码\n",
    "        shape_codes = list()\n",
    "\n",
    "        for word_i in words:\n",
    "            # 形体结构  1 位,  字典中无此字符的用 '0' 代替\n",
    "            structure_code = self.hz_structure_dict.get(word_i, '0')\n",
    "\n",
    "            # 四角编码  5 位,  字典中无此字符的用 '00000' 代替\n",
    "            fourcorner_code = self.hz_fourcorner_dict.get(word_i, '00000')\n",
    "\n",
    "            # 笔画数编码 1 位,  字典中无此字符的用 '0' 代替\n",
    "            strokes_code = self.hz_strokes_dict.get(word_i, '0')\n",
    "\n",
    "            # print('{} {} {}'.format(structure_code, fourcorner_code, strokes_code))\n",
    "\n",
    "            shape_codes.append(structure_code + fourcorner_code + strokes_code)\n",
    "\n",
    "        return shape_codes\n",
    "\n",
    "\n",
    "# 汉字相似度计算\n",
    "class WordsSimilarity(object):\n",
    "    \"\"\"\n",
    "    汉字相似度【拼音+字形】计算\n",
    "\n",
    "        德:['523', '124231F']  的:['503', '1276208']  SoundScore:0.800  ShapeScore:0.483\n",
    "        True\n",
    "        来:['827', 'D509007']  未:['N4j', 'D509005']  SoundScore:0.000  ShapeScore:0.929\n",
    "        True\n",
    "\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, structure_file_path, fourcorner_file_path, strokes_file_path,\n",
    "                 sound_code_weight=(0.4, 0.2, 0.4), shape_code_weight=(0.25, 0.1, 0.1, 0.1, 0.1, 0.1, 0.25),\n",
    "                 sound_shape_threshold=(0.5, 0.5)):\n",
    "        \"\"\"\n",
    "        :param structure_file_path:    汉字结构编码表\n",
    "        :param fourcorner_file_path:   汉字四角编码表\n",
    "        :param strokes_file_path:      汉字笔画编码表\n",
    "        :param sound_code_weight:  【声母（1位）】 + 【声调（1位）】 + 【韵母（1位）】  各个权重。 权重和为 1\n",
    "        :param shape_code_weight: 【结构编码（1位）】 + 【四角编码 （5位）】 + 【笔划编码（1位）】  各个权重。 权重和为 1\n",
    "        :param sound_shape_threshold:   【拼音编码】 + 【字形编码】  阈值。\n",
    "                (0.8, 0.5)   拼音相似度 > 0.8   AND  字形相似度 > 0.5\n",
    "                (-0.8, -0.95)   拼音相似度 > 0.8  OR  字形相似度 > 0.95\n",
    "        \"\"\"\n",
    "        # 拼音编码相似度 权重\n",
    "        self.sound_code_weight = sound_code_weight\n",
    "        # 字形编码相似度 权重\n",
    "        self.shape_code_weight = shape_code_weight\n",
    "\n",
    "        # 拼音、字形相似度阈值\n",
    "        self.sound_shape_threshold = sound_shape_threshold\n",
    "\n",
    "        # 拼音编码获取类 初始化\n",
    "        self.sound_code = SoundCode()\n",
    "\n",
    "        # 字形编码获取类 初始化\n",
    "        self.shape_code = ShapeCode(structure_file_path, fourcorner_file_path, strokes_file_path)\n",
    "\n",
    "        # 笔划编码 逆字典\n",
    "        self.strokes_dict_reverse = {v: k for k, v in self.shape_code.strokes_num_dict.items()}\n",
    "\n",
    "        # Match 函数使用\n",
    "        self.match_next_val = [-1]\n",
    "        self.match_start_idxRes = []\n",
    "\n",
    "    # 拼音相似度 计算\n",
    "    def sound_code_similarity(self, sc_1, sc_2):\n",
    "        \"\"\"\n",
    "        :param sc_1:     第一位拼音码 '523'\n",
    "        :param sc_2:     第二位拼音码 '503'\n",
    "        :return:   相似度得分 【0, 1】\n",
    "        \"\"\"\n",
    "        assert len(sc_1) == len(self.sound_code_weight)\n",
    "        assert len(sc_2) == len(self.sound_code_weight)\n",
    "\n",
    "        similar_score = 0  # 相似度得分\n",
    "\n",
    "        # 非法字符直接返回0\n",
    "        if sc_1 == '000' or sc_2 == '000':\n",
    "            return similar_score\n",
    "\n",
    "        for sc_1_i, sc_2_i, w_i in zip(sc_1, sc_2, self.sound_code_weight):\n",
    "            if sc_1_i == sc_2_i:\n",
    "                similar_score += w_i\n",
    "\n",
    "        return similar_score\n",
    "\n",
    "    # 字形相似度 计算\n",
    "    def shape_code_similarity(self, sc_1, sc_2):\n",
    "        \"\"\"\n",
    "        :param sc_1:     第一位字形码 'D509005'\n",
    "        :param sc_2:     第二位字形码 'D509007'\n",
    "        :return:   相似度得分 【0, 1】\n",
    "        \"\"\"\n",
    "        assert len(sc_1) == len(self.shape_code_weight)\n",
    "        assert len(sc_2) == len(self.shape_code_weight)\n",
    "\n",
    "        similar_score = 0  # 相似度得分\n",
    "\n",
    "        # 非法字符直接返回0\n",
    "        if sc_1 == '0000000' or sc_2 == '0000000':\n",
    "            return similar_score\n",
    "\n",
    "        # 结构编码、四角编码\n",
    "        for i, w_i in enumerate(self.shape_code_weight[:-1]):\n",
    "            if sc_1[i] == sc_2[i]:\n",
    "                similar_score += w_i\n",
    "\n",
    "        # 有笔划数 为 0 的异常汉字 其笔划相似度得分为 0\n",
    "        if sc_1[-1] == 0 or sc_2[-1] == 0:\n",
    "            return similar_score\n",
    "\n",
    "        # 笔划编码  两字笔划数量差 占比\n",
    "        similar_score += (1 - abs(self.strokes_dict_reverse[sc_1[-1]] - self.strokes_dict_reverse[sc_2[-1]]) * 1.0\n",
    "                          / max(self.strokes_dict_reverse[sc_1[-1]], self.strokes_dict_reverse[sc_2[-1]])) * \\\n",
    "                         self.shape_code_weight[-1]\n",
    "\n",
    "        return similar_score\n",
    "\n",
    "    # 拼音相似度 + 字形相似度\n",
    "    def single(self, w_1, w_2, debug=False):\n",
    "        \"\"\"\n",
    "        :param w_1:    汉字 1\n",
    "        :param w_2:    汉字 2\n",
    "        :param debug:  debug\n",
    "        :return:\n",
    "        \"\"\"\n",
    "        \"\"\"\n",
    "        :param sc_1:     第一位拼音码  '523' +  字形码 'D509005'\n",
    "        :param sc_2:     第二位拼音码  '503' +  字形码 'D509007'\n",
    "        :return: \n",
    "        \"\"\"\n",
    "        # 字形编码\n",
    "        shape_code_1 = self.shape_code(w_1)[0]\n",
    "        shape_code_2 = self.shape_code(w_2)[0]\n",
    "        # 字形相似度\n",
    "        shape_code_score = self.shape_code_similarity(shape_code_1, shape_code_2)\n",
    "\n",
    "        # 拼音编码\n",
    "        sound_code_1 = self.sound_code(w_1)[0]\n",
    "        sound_code_2 = self.sound_code(w_2)[0]\n",
    "        # 拼音相似度\n",
    "        sound_code_score = self.sound_code_similarity(sound_code_1, sound_code_2)\n",
    "\n",
    "        if debug:\n",
    "            print('{}:{}  {}:{}  SoundScore:{:.3f}  ShapeScore:{:.3f}'.format(\n",
    "                w_1, sound_code_1 + shape_code_1, w_2, sound_code_2 + shape_code_2, sound_code_score, shape_code_score))\n",
    "\n",
    "        # 设置阈值 >= 0   AND 逻辑\n",
    "        if self.sound_shape_threshold[0] >= 0 and self.sound_shape_threshold[1] >= 0:\n",
    "            return sound_code_score >= self.sound_shape_threshold[0] and \\\n",
    "                   shape_code_score >= self.sound_shape_threshold[1]\n",
    "\n",
    "        # 设置阈值 < 0   OR 逻辑\n",
    "        if self.sound_shape_threshold[0] < 0 and self.sound_shape_threshold[1] < 0:\n",
    "            return sound_code_score >= -1 * self.sound_shape_threshold[0] or \\\n",
    "                   shape_code_score >= -1 * self.sound_shape_threshold[1]\n",
    "\n",
    "    # 相似度计算\n",
    "    def similar_score(self, code_1, code_2, debug=False):\n",
    "\n",
    "        # 拼音编码\n",
    "        sound_code_1 = code_1[:3]\n",
    "        sound_code_2 = code_2[:3]\n",
    "        # 拼音相似度\n",
    "        sound_code_score = self.sound_code_similarity(sound_code_1, sound_code_2)\n",
    "\n",
    "        # 形状编码\n",
    "        shape_code_1 = code_1[3:]\n",
    "        shape_code_2 = code_2[3:]\n",
    "        # 字形相似度\n",
    "        shape_code_score = self.shape_code_similarity(shape_code_1, shape_code_2)\n",
    "\n",
    "        if debug:\n",
    "            print('{}  {}  SoundScore:{:.3f}  ShapeScore:{:.3f}'.format(\n",
    "                sound_code_1 + shape_code_1, sound_code_2 + shape_code_2, sound_code_score, shape_code_score))\n",
    "\n",
    "        # 设置阈值 >= 0   AND 逻辑\n",
    "        if self.sound_shape_threshold[0] >= 0 and self.sound_shape_threshold[1] >= 0:\n",
    "            return sound_code_score >= self.sound_shape_threshold[0] and \\\n",
    "                   shape_code_score >= self.sound_shape_threshold[1]\n",
    "\n",
    "        # 设置阈值 < 0   OR 逻辑\n",
    "        if self.sound_shape_threshold[0] < 0 and self.sound_shape_threshold[1] < 0:\n",
    "            return sound_code_score >= -1 * self.sound_shape_threshold[0] or \\\n",
    "                   shape_code_score >= -1 * self.sound_shape_threshold[1]\n",
    "\n",
    "    # 字符串匹配搜索\n",
    "    def match(self, words_1, words_2, debug=False):\n",
    "        \"\"\"\n",
    "        :param words_1:   目标词  【紫琅路】\n",
    "        :param words_2:   待检索句子  【国我爱你女生于无娃哇紫狼路爽晕约紫薇路又刘页列而紫粮路掩连哟罗】\n",
    "        :param debug:\n",
    "        :return:  返回目标词在待检索句子中的位置索引\n",
    "                        Index:10  MatchWord:紫狼路\n",
    "                        Index:24  MatchWord:紫粮路\n",
    "        \"\"\"\n",
    "        self.match_next_val = [-1]\n",
    "        self.match_start_idxRes = []\n",
    "\n",
    "        # 拼音编码\n",
    "        sound_code_1 = self.sound_code(words_1)\n",
    "        sound_code_2 = self.sound_code(words_2)\n",
    "\n",
    "        # 字形编码\n",
    "        shape_code_1 = self.shape_code(words_1)\n",
    "        shape_code_2 = self.shape_code(words_2)\n",
    "\n",
    "        needle = [sound_code_i + shape_code_i for sound_code_i, shape_code_i in zip(sound_code_1, shape_code_1)]\n",
    "\n",
    "        haystack = [sound_code_i + shape_code_i for sound_code_i, shape_code_i in zip(sound_code_2, shape_code_2)]\n",
    "\n",
    "        # 子串定位，即模式匹配，可采用BF算法  也可采用KMP算法，我采用KMP算法\n",
    "        # 0<= pos <= len(strS) - len(strT)) + 1\n",
    "        self._get_next_val(needle)\n",
    "        i = 0\n",
    "        while i < len(haystack):\n",
    "            j = 0\n",
    "            while i < len(haystack) and j < len(needle):\n",
    "                # if j == -1 or haystack[i] == needle[j]:\n",
    "                if j == -1 or self.similar_score(haystack[i], needle[j]):\n",
    "                    i += 1\n",
    "                    j += 1\n",
    "                else:\n",
    "                    j = self.match_next_val[j]\n",
    "            if j == len(needle):\n",
    "                self.match_start_idxRes.append(i - len(needle))\n",
    "\n",
    "        return self.match_start_idxRes\n",
    "\n",
    "    def _get_next_val(self, str_t):\n",
    "        i = 0\n",
    "        j = -1\n",
    "        while i < len(str_t) - 1:\n",
    "            # if j == -1 or strT[i] == strT[j]:\n",
    "            if j == -1 or self.similar_score(str_t[i], str_t[j]):\n",
    "                i += 1\n",
    "                j += 1\n",
    "                # if i < len(strT) and strT[i] == strT[j]:\n",
    "                if i < len(str_t) and self.similar_score(str_t[i], str_t[j]):\n",
    "                    self.match_next_val.append(self.match_next_val[j])\n",
    "                else:\n",
    "                    self.match_next_val.append(j)\n",
    "            else:\n",
    "                j = self.match_next_val[j]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 初始化 类\n",
    "ws = WordsSimilarity(structure_file_path=os.path.join('.', 'zh_data', 'Structure.txt'),\n",
    "                     fourcorner_file_path=os.path.join('.', 'zh_data', 'FourCorner.txt'),\n",
    "                     strokes_file_path=os.path.join('.', 'zh_data', 'Strokes.txt'),\n",
    "                     sound_shape_threshold=(-0.7, -0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "德:523124231F  的:5031276208  SoundScore:0.800  ShapeScore:0.483\n",
      "True\n",
      "来:827D509007  未:N4jD509005  SoundScore:0.000  ShapeScore:0.929\n",
      "True\n",
      "☆:0000000000  来:827D509007  SoundScore:0.000  ShapeScore:0.000\n",
      "False\n",
      "☆:0000000000  *:0000000000  SoundScore:0.000  ShapeScore:0.000\n",
      "False\n",
      "琅:82k113132B  狼:82k143232A  SoundScore:1.000  ShapeScore:0.777\n",
      "True\n",
      "琅:82k113132B  粮:82l193932D  SoundScore:0.600  ShapeScore:0.762\n",
      "True\n",
      "谁:L28130715A  睡:L4j162015D  SoundScore:0.400  ShapeScore:0.642\n",
      "False\n",
      "160 µs ± 8.45 µs per loop (mean ± std. dev. of 7 runs, 10000 loops each)\n"
     ]
    }
   ],
   "source": [
    "# 汉字相似度 计算\n",
    "print(ws.single('德', '的', debug=True))\n",
    "print(ws.single('来', '未', debug=True))\n",
    "print(ws.single('☆', '来', debug=True))\n",
    "print(ws.single('☆', '*', debug=True))\n",
    "print(ws.single('琅', '狼', debug=True))\n",
    "print(ws.single('琅', '粮', debug=True))\n",
    "print(ws.single('谁', '睡', debug=True))\n",
    "\n",
    "# 拼音部分有误\n",
    "# 谁 ['sh'] ['ei2']\n",
    "# 睡 ['sh'] ['uei4']\n",
    "\n",
    "%timeit ws.single('谁', '睡', debug=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index:0  MatchWord:刘的花的老婆\n",
      "833 µs ± 93.1 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n"
     ]
    }
   ],
   "source": [
    "chi_word1 = '紫琅路'\n",
    "chi_word2 = '国我爱你女生于无娃哇紫狼路爽晕约紫薇路又刘页列而紫粮路掩连哟罗'\n",
    "\n",
    "chi_word1 = '留得华的老婆'\n",
    "chi_word2 = '刘的花的老婆是XXX'\n",
    "\n",
    "# 模糊匹配\n",
    "match_index = ws.match(chi_word1, chi_word2, debug=True)\n",
    "for index_i in match_index:\n",
    "    print('Index:{}  MatchWord:{}'.format(index_i, chi_word2[index_i: index_i+len(chi_word1)]))\n",
    "    \n",
    "%timeit match_index = ws.match(chi_word1, chi_word2, debug=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
